店播爬取Python脚本

proto_builder.py 5.4KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138
  1. # Protocol Buffers - Google's data interchange format
  2. # Copyright 2008 Google Inc. All rights reserved.
  3. # https://developers.google.com/protocol-buffers/
  4. #
  5. # Redistribution and use in source and binary forms, with or without
  6. # modification, are permitted provided that the following conditions are
  7. # met:
  8. #
  9. # * Redistributions of source code must retain the above copyright
  10. # notice, this list of conditions and the following disclaimer.
  11. # * Redistributions in binary form must reproduce the above
  12. # copyright notice, this list of conditions and the following disclaimer
  13. # in the documentation and/or other materials provided with the
  14. # distribution.
  15. # * Neither the name of Google Inc. nor the names of its
  16. # contributors may be used to endorse or promote products derived from
  17. # this software without specific prior written permission.
  18. #
  19. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20. # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21. # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22. # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23. # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24. # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25. # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26. # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27. # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28. # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30. """Dynamic Protobuf class creator."""
  31. try:
  32. from collections import OrderedDict
  33. except ImportError:
  34. from ordereddict import OrderedDict #PY26
  35. import hashlib
  36. import os
  37. from google.protobuf import descriptor_pb2
  38. from google.protobuf import descriptor
  39. from google.protobuf import message_factory
  40. def _GetMessageFromFactory(factory, full_name):
  41. """Get a proto class from the MessageFactory by name.
  42. Args:
  43. factory: a MessageFactory instance.
  44. full_name: str, the fully qualified name of the proto type.
  45. Returns:
  46. A class, for the type identified by full_name.
  47. Raises:
  48. KeyError, if the proto is not found in the factory's descriptor pool.
  49. """
  50. proto_descriptor = factory.pool.FindMessageTypeByName(full_name)
  51. proto_cls = factory.GetPrototype(proto_descriptor)
  52. return proto_cls
  53. def MakeSimpleProtoClass(fields, full_name=None, pool=None):
  54. """Create a Protobuf class whose fields are basic types.
  55. Note: this doesn't validate field names!
  56. Args:
  57. fields: dict of {name: field_type} mappings for each field in the proto. If
  58. this is an OrderedDict the order will be maintained, otherwise the
  59. fields will be sorted by name.
  60. full_name: optional str, the fully-qualified name of the proto type.
  61. pool: optional DescriptorPool instance.
  62. Returns:
  63. a class, the new protobuf class with a FileDescriptor.
  64. """
  65. factory = message_factory.MessageFactory(pool=pool)
  66. if full_name is not None:
  67. try:
  68. proto_cls = _GetMessageFromFactory(factory, full_name)
  69. return proto_cls
  70. except KeyError:
  71. # The factory's DescriptorPool doesn't know about this class yet.
  72. pass
  73. # Get a list of (name, field_type) tuples from the fields dict. If fields was
  74. # an OrderedDict we keep the order, but otherwise we sort the field to ensure
  75. # consistent ordering.
  76. field_items = fields.items()
  77. if not isinstance(fields, OrderedDict):
  78. field_items = sorted(field_items)
  79. # Use a consistent file name that is unlikely to conflict with any imported
  80. # proto files.
  81. fields_hash = hashlib.sha1()
  82. for f_name, f_type in field_items:
  83. fields_hash.update(f_name.encode('utf-8'))
  84. fields_hash.update(str(f_type).encode('utf-8'))
  85. proto_file_name = fields_hash.hexdigest() + '.proto'
  86. # If the proto is anonymous, use the same hash to name it.
  87. if full_name is None:
  88. full_name = ('net.proto2.python.public.proto_builder.AnonymousProto_' +
  89. fields_hash.hexdigest())
  90. try:
  91. proto_cls = _GetMessageFromFactory(factory, full_name)
  92. return proto_cls
  93. except KeyError:
  94. # The factory's DescriptorPool doesn't know about this class yet.
  95. pass
  96. # This is the first time we see this proto: add a new descriptor to the pool.
  97. factory.pool.Add(
  98. _MakeFileDescriptorProto(proto_file_name, full_name, field_items))
  99. return _GetMessageFromFactory(factory, full_name)
  100. def _MakeFileDescriptorProto(proto_file_name, full_name, field_items):
  101. """Populate FileDescriptorProto for MessageFactory's DescriptorPool."""
  102. package, name = full_name.rsplit('.', 1)
  103. file_proto = descriptor_pb2.FileDescriptorProto()
  104. file_proto.name = os.path.join(package.replace('.', '/'), proto_file_name)
  105. file_proto.package = package
  106. desc_proto = file_proto.message_type.add()
  107. desc_proto.name = name
  108. for f_number, (f_name, f_type) in enumerate(field_items, 1):
  109. field_proto = desc_proto.field.add()
  110. field_proto.name = f_name
  111. # # If the number falls in the reserved range, reassign it to the correct
  112. # # number after the range.
  113. if f_number >= descriptor.FieldDescriptor.FIRST_RESERVED_FIELD_NUMBER:
  114. f_number += (
  115. descriptor.FieldDescriptor.LAST_RESERVED_FIELD_NUMBER -
  116. descriptor.FieldDescriptor.FIRST_RESERVED_FIELD_NUMBER + 1)
  117. field_proto.number = f_number
  118. field_proto.label = descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL
  119. field_proto.type = f_type
  120. return file_proto