店播爬取Python脚本

symbol_database.py 6.8KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195
  1. # Protocol Buffers - Google's data interchange format
  2. # Copyright 2008 Google Inc. All rights reserved.
  3. # https://developers.google.com/protocol-buffers/
  4. #
  5. # Redistribution and use in source and binary forms, with or without
  6. # modification, are permitted provided that the following conditions are
  7. # met:
  8. #
  9. # * Redistributions of source code must retain the above copyright
  10. # notice, this list of conditions and the following disclaimer.
  11. # * Redistributions in binary form must reproduce the above
  12. # copyright notice, this list of conditions and the following disclaimer
  13. # in the documentation and/or other materials provided with the
  14. # distribution.
  15. # * Neither the name of Google Inc. nor the names of its
  16. # contributors may be used to endorse or promote products derived from
  17. # this software without specific prior written permission.
  18. #
  19. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20. # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21. # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22. # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23. # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24. # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25. # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26. # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27. # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28. # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30. """A database of Python protocol buffer generated symbols.
  31. SymbolDatabase is the MessageFactory for messages generated at compile time,
  32. and makes it easy to create new instances of a registered type, given only the
  33. type's protocol buffer symbol name.
  34. Example usage::
  35. db = symbol_database.SymbolDatabase()
  36. # Register symbols of interest, from one or multiple files.
  37. db.RegisterFileDescriptor(my_proto_pb2.DESCRIPTOR)
  38. db.RegisterMessage(my_proto_pb2.MyMessage)
  39. db.RegisterEnumDescriptor(my_proto_pb2.MyEnum.DESCRIPTOR)
  40. # The database can be used as a MessageFactory, to generate types based on
  41. # their name:
  42. types = db.GetMessages(['my_proto.proto'])
  43. my_message_instance = types['MyMessage']()
  44. # The database's underlying descriptor pool can be queried, so it's not
  45. # necessary to know a type's filename to be able to generate it:
  46. filename = db.pool.FindFileContainingSymbol('MyMessage')
  47. my_message_instance = db.GetMessages([filename])['MyMessage']()
  48. # This functionality is also provided directly via a convenience method:
  49. my_message_instance = db.GetSymbol('MyMessage')()
  50. """
  51. from google.protobuf.internal import api_implementation
  52. from google.protobuf import descriptor_pool
  53. from google.protobuf import message_factory
  54. class SymbolDatabase(message_factory.MessageFactory):
  55. """A database of Python generated symbols."""
  56. def RegisterMessage(self, message):
  57. """Registers the given message type in the local database.
  58. Calls to GetSymbol() and GetMessages() will return messages registered here.
  59. Args:
  60. message: A :class:`google.protobuf.message.Message` subclass (or
  61. instance); its descriptor will be registered.
  62. Returns:
  63. The provided message.
  64. """
  65. desc = message.DESCRIPTOR
  66. self._classes[desc] = message
  67. self.RegisterMessageDescriptor(desc)
  68. return message
  69. def RegisterMessageDescriptor(self, message_descriptor):
  70. """Registers the given message descriptor in the local database.
  71. Args:
  72. message_descriptor (Descriptor): the message descriptor to add.
  73. """
  74. if api_implementation.Type() == 'python':
  75. # pylint: disable=protected-access
  76. self.pool._AddDescriptor(message_descriptor)
  77. def RegisterEnumDescriptor(self, enum_descriptor):
  78. """Registers the given enum descriptor in the local database.
  79. Args:
  80. enum_descriptor (EnumDescriptor): The enum descriptor to register.
  81. Returns:
  82. EnumDescriptor: The provided descriptor.
  83. """
  84. if api_implementation.Type() == 'python':
  85. # pylint: disable=protected-access
  86. self.pool._AddEnumDescriptor(enum_descriptor)
  87. return enum_descriptor
  88. def RegisterServiceDescriptor(self, service_descriptor):
  89. """Registers the given service descriptor in the local database.
  90. Args:
  91. service_descriptor (ServiceDescriptor): the service descriptor to
  92. register.
  93. """
  94. if api_implementation.Type() == 'python':
  95. # pylint: disable=protected-access
  96. self.pool._AddServiceDescriptor(service_descriptor)
  97. def RegisterFileDescriptor(self, file_descriptor):
  98. """Registers the given file descriptor in the local database.
  99. Args:
  100. file_descriptor (FileDescriptor): The file descriptor to register.
  101. """
  102. if api_implementation.Type() == 'python':
  103. # pylint: disable=protected-access
  104. self.pool._InternalAddFileDescriptor(file_descriptor)
  105. def GetSymbol(self, symbol):
  106. """Tries to find a symbol in the local database.
  107. Currently, this method only returns message.Message instances, however, if
  108. may be extended in future to support other symbol types.
  109. Args:
  110. symbol (str): a protocol buffer symbol.
  111. Returns:
  112. A Python class corresponding to the symbol.
  113. Raises:
  114. KeyError: if the symbol could not be found.
  115. """
  116. return self._classes[self.pool.FindMessageTypeByName(symbol)]
  117. def GetMessages(self, files):
  118. # TODO(amauryfa): Fix the differences with MessageFactory.
  119. """Gets all registered messages from a specified file.
  120. Only messages already created and registered will be returned; (this is the
  121. case for imported _pb2 modules)
  122. But unlike MessageFactory, this version also returns already defined nested
  123. messages, but does not register any message extensions.
  124. Args:
  125. files (list[str]): The file names to extract messages from.
  126. Returns:
  127. A dictionary mapping proto names to the message classes.
  128. Raises:
  129. KeyError: if a file could not be found.
  130. """
  131. def _GetAllMessages(desc):
  132. """Walk a message Descriptor and recursively yields all message names."""
  133. yield desc
  134. for msg_desc in desc.nested_types:
  135. for nested_desc in _GetAllMessages(msg_desc):
  136. yield nested_desc
  137. result = {}
  138. for file_name in files:
  139. file_desc = self.pool.FindFileByName(file_name)
  140. for msg_desc in file_desc.message_types_by_name.values():
  141. for desc in _GetAllMessages(msg_desc):
  142. try:
  143. result[desc.full_name] = self._classes[desc]
  144. except KeyError:
  145. # This descriptor has no registered class, skip it.
  146. pass
  147. return result
  148. _DEFAULT = SymbolDatabase(pool=descriptor_pool.Default())
  149. def Default():
  150. """Returns the default SymbolDatabase."""
  151. return _DEFAULT