wget https://www.python.org/ftp/python/2.7.10/Python-2.7.10.tgz

tar -zxvf Python-2.7..tgz

cd Python-2.7.

./configure

make all

make installmake clean

make distclean  

python --version

mv /usr/bin/python /usr/bin/python2..6_bak

ln -s /usr/local/bin/python2. /usr/bin/python

# python --version

Python 2.7.

-bash: pip: command not found

wget https://bootstrap.pypa.io/get-pip.py

python get-pip.py

Collecting Twisted>=10.0. (from scrapy)

  Could not find a version that satisfies the requirement Twisted>=10.0. (from scrapy) (from versions: )

No matching distribution found for Twisted>=10.0. (from scrapy)

wget https://pypi.python.org/packages/source/T/Twisted/Twisted-15.2.1.tar.bz2
tar -xjvf Twisted-15.2.1.tar.bz2
cd Twisted-15.2.1
python setup.py install

python

Python 2.7. (default, Jun   , ::)

[GCC 4.4.  (Red Hat 4.4.-)] on linux2

Type "help", "copyright", "credits" or "license" for more information.

>>> import twisted

>>> 

Collecting libxlst

  Could not find a version that satisfies the requirement libxlst (from versions: )

No matching distribution found for libxlst

Collecting libxml2

  Could not find a version that satisfies the requirement libxml2 (from versions: )

No matching distribution found for libxml2

wget http://xmlsoft.org/sources/libxslt-1.1.28.tar.gz
tar -zxvf libxslt-1.1.28.tar.gz

cd libxslt-1.1./

./configure

make

make install

wget ftp://xmlsoft.org/libxml2/libxml2-git-snapshot.tar.gz
tar -zxvf libxml2-git-snapshot.tar.gz

cd libxml2-2.9./

./configure

make

make install

Failed building wheel for cryptography

wget https://pypi.python.org/packages/source/c/cryptography/cryptography-0.4.tar.gz
tar -zxvf cryptography-0.4.tar.gz
cd cryptography-0.4
python setup.py build
python setup.py install

No package 'libffi' found

wget ftp://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz

tar -zxvf libffi-3.2.1.tar.gz
cd libffi-3.2.1
./configure

make

make install

Package libffi was not found in the pkg-config search path.

    Perhaps you should add the directory containing `libffi.pc'

    to the PKG_CONFIG_PATH environment variable

    No package 'libffi' found

export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH

pip install scrapy

ImportError: libffi.so.: cannot open shared object file: No such file or directory

whereis libffi
libffi: /usr/local/lib/libffi.a /usr/local/lib/libffi.la /usr/local/lib/libffi.so

export LD_LIBRARY_PATH=/usr/local/lib

python setup.py build

python setup.py install

pip install scrapy

Building wheels for collected packages: cryptography

  Running setup.py bdist_wheel for cryptography

Requirement already satisfied (use --upgrade to upgrade): zope.interface>=3.6. in /usr/local/lib/python2./site-packages/zope.interface-4.1.-py2.-linux-i686.egg (from Twisted>=10.0.->scrapy)

Collecting cryptography>=0.7 (from pyOpenSSL->scrapy)

  Using cached cryptography-0.9.tar.gz

Requirement already satisfied (use --upgrade to upgrade): setuptools in /usr/local/lib/python2./site-packages (from zope.interface>=3.6.->Twisted>=10.0.->scrapy)

Requirement already satisfied (use --upgrade to upgrade): idna in /usr/local/lib/python2./site-packages (from cryptography>=0.7->pyOpenSSL->scrapy)

Requirement already satisfied (use --upgrade to upgrade): pyasn1 in /usr/local/lib/python2./site-packages (from cryptography>=0.7->pyOpenSSL->scrapy)

Requirement already satisfied (use --upgrade to upgrade): enum34 in /usr/local/lib/python2./site-packages (from cryptography>=0.7->pyOpenSSL->scrapy)

Requirement already satisfied (use --upgrade to upgrade): ipaddress in /usr/local/lib/python2./site-packages (from cryptography>=0.7->pyOpenSSL->scrapy)

Requirement already satisfied (use --upgrade to upgrade): cffi>=0.8 in /usr/local/lib/python2./site-packages (from cryptography>=0.7->pyOpenSSL->scrapy)

Requirement already satisfied (use --upgrade to upgrade): ordereddict in /usr/local/lib/python2./site-packages (from enum34->cryptography>=0.7->pyOpenSSL->scrapy)

Requirement already satisfied (use --upgrade to upgrade): pycparser in /usr/local/lib/python2./site-packages (from cffi>=0.8->cryptography>=0.7->pyOpenSSL->scrapy)

Building wheels for collected packages: cryptography

  Running setup.py bdist_wheel for cryptography

  Stored in directory: /root/.cache/pip/wheels/d7///7258f08eae0b9c930c04209959c9a0794b9729c2b64258117e

Successfully built cryptography

Installing collected packages: cryptography

  Found existing installation: cryptography 0.4

    Uninstalling cryptography-0.4:

      Successfully uninstalled cryptography-0.4

Successfully installed cryptography-0.9

cat > myspider.py <<EOF

from scrapy import Spider, Item, Field

class Post(Item):

    title = Field()

class BlogSpider(Spider):

    name, start_urls = 'blogspider', ['http://www.cnblogs.com/rwxwsblog/']

    def parse(self, response):

        return [Post(title=e.extract()) for e in response.css("h2 a::text")]

EOF

scrapy runspider myspider.py

-- :: [scrapy] INFO: Scrapy 1.0.0rc2 started (bot: scrapybot)

-- :: [scrapy] INFO: Optional features available: ssl, http11

-- :: [scrapy] INFO: Overridden settings: {}

-- :: [py.warnings] WARNING: :: UserWarning: You do not have a working installation of the service_identity module: 'No module named service_identity'.  Please install it from <https://pypi.python.org/pypi/service_identity> and make sure all of its dependencies are satisfied.  Without the service_identity module and a recent enough pyOpenSSL to support it, Twisted can perform only rudimentary TLS client hostname verification.  Many valid certificate/hostname mappings may be rejected.

-- :: [scrapy] INFO: Enabled extensions: CloseSpider, TelnetConsole, LogStats, CoreStats, SpiderState

-- :: [scrapy] INFO: Enabled downloader middlewares: HttpAuthMiddleware, DownloadTimeoutMiddleware, UserAgentMiddleware, RetryMiddleware, DefaultHeadersMiddleware, MetaRefreshMiddleware, HttpCompressionMiddleware, RedirectMiddleware, CookiesMiddleware, ChunkedTransferMiddleware, DownloaderStats

-- :: [scrapy] INFO: Enabled spider middlewares: HttpErrorMiddleware, OffsiteMiddleware, RefererMiddleware, UrlLengthMiddleware, DepthMiddleware

-- :: [scrapy] INFO: Enabled item pipelines:

-- :: [scrapy] INFO: Spider opened

-- :: [scrapy] INFO: Crawled  pages (at  pages/min), scraped  items (at  items/min)

-- :: [scrapy] DEBUG: Telnet console listening on 127.0.0.1:

-- :: [scrapy] DEBUG: Crawled () <GET http://www.cnblogs.com/rwxwsblog/> (referer: None)

-- :: [scrapy] INFO: Closing spider (finished)

-- :: [scrapy] INFO: Dumping Scrapy stats:

{'downloader/request_bytes': ,

 'downloader/request_count': ,

 'downloader/request_method_count/GET': ,

 'downloader/response_bytes': ,

 'downloader/response_count': ,

 'downloader/response_status_count/200': ,

 'finish_reason': 'finished',

 'finish_time': datetime.datetime(, , , , , , ),

 'log_count/DEBUG': ,

 'log_count/INFO': ,

 'log_count/WARNING': ,

 'response_received_count': ,

 'scheduler/dequeued': ,

 'scheduler/dequeued/memory': ,

 'scheduler/enqueued': ,

 'scheduler/enqueued/memory': ,

 'start_time': datetime.datetime(, , , , , , )}

-- :: [scrapy] INFO: Spider closed (finished)

scrapy startproject tutorial

Traceback (most recent call last):

  File "/usr/local/bin/scrapy", line , in <module>

    load_entry_point('Scrapy==1.0.0rc2', 'console_scripts', 'scrapy')()

  File "/usr/local/lib/python2.7/site-packages/pkg_resources/__init__.py", line , in load_entry_point

    return get_distribution(dist).load_entry_point(group, name)

  File "/usr/local/lib/python2.7/site-packages/pkg_resources/__init__.py", line , in load_entry_point

    return ep.load()

  File "/usr/local/lib/python2.7/site-packages/pkg_resources/__init__.py", line , in load

    return self.resolve()

  File "/usr/local/lib/python2.7/site-packages/pkg_resources/__init__.py", line , in resolve

    module = __import__(self.module_name, fromlist=['__name__'], level=)

  File "/usr/local/lib/python2.7/site-packages/Scrapy-1.0.0rc2-py2.7.egg/scrapy/__init__.py", line , in <module>

    from scrapy.spiders import Spider

  File "/usr/local/lib/python2.7/site-packages/Scrapy-1.0.0rc2-py2.7.egg/scrapy/spiders/__init__.py", line , in <module>

    from scrapy.http import Request

  File "/usr/local/lib/python2.7/site-packages/Scrapy-1.0.0rc2-py2.7.egg/scrapy/http/__init__.py", line , in <module>

    from scrapy.http.request.form import FormRequest

  File "/usr/local/lib/python2.7/site-packages/Scrapy-1.0.0rc2-py2.7.egg/scrapy/http/request/form.py", line , in <module>

    import lxml.html

  File "/usr/local/lib/python2.7/site-packages/lxml/html/__init__.py", line , in <module>

    from lxml import etree

ImportError: /usr/lib/libxml2.so.: version `LIBXML2_2.9.0' not found (required by /usr/local/lib/python2.7/site-packages/lxml/etree.so)

export LD_LIBRARY_PATH=/usr/local/lib

scrapy startproject tutorial

[root@bogon scrapy]# scrapy startproject tutorial

-- :: [scrapy] INFO: Scrapy 1.0.0rc2 started (bot: scrapybot)

-- :: [scrapy] INFO: Optional features available: ssl, http11

-- :: [scrapy] INFO: Overridden settings: {}

New Scrapy project 'tutorial' created in:

    /root/scrapy/tutorial

You can start your first spider with:

    cd tutorial

    scrapy genspider example example.com

vi /etc/profile

libtool: install: /usr/bin/install -c .libs/libffi.so.6.0. /usr/local/lib/../lib64/libffi.so.6.0.

libtool: install: (cd /usr/local/lib/../lib64 && { ln -s -f libffi.so.6.0. libffi.so. || { rm -f libffi.so. && ln -s libffi.so.6.0. libffi.so.; }; })

libtool: install: (cd /usr/local/lib/../lib64 && { ln -s -f libffi.so.6.0. libffi.so || { rm -f libffi.so && ln -s libffi.so.6.0. libffi.so; }; })

libtool: install: /usr/bin/install -c .libs/libffi.lai /usr/local/lib/../lib64/libffi.la

libtool: install: /usr/bin/install -c .libs/libffi.a /usr/local/lib/../lib64/libffi.a

libtool: install: chmod  /usr/local/lib/../lib64/libffi.a

libtool: install: ranlib /usr/local/lib/../lib64/libffi.a

libtool: finish: PATH="/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/www/wdlinux/mysql/bin:/root/bin:/sbin" ldconfig -n /usr/local/lib/../lib64

----------------------------------------------------------------------

Libraries have been installed in:

   /usr/local/lib/../lib64

If you ever happen to want to link against installed libraries

in a given directory, LIBDIR, you must either use libtool, and

specify the full pathname of the library, or use the `-LLIBDIR'

flag during linking and do at least one of the following:

   - add LIBDIR to the `LD_LIBRARY_PATH' environment variable

     during execution

   - add LIBDIR to the `LD_RUN_PATH' environment variable

     during linking

   - use the `-Wl,-rpath -Wl,LIBDIR' linker flag

   - have your system administrator add LIBDIR to `/etc/ld.so.conf'

See any operating system documentation about shared libraries for

more information, such as the ld() and ld.so() manual pages.

----------------------------------------------------------------------

 /bin/mkdir -p '/usr/local/share/info'

 /usr/bin/install -c -m  ../doc/libffi.info '/usr/local/share/info'

 install-info --info-dir='/usr/local/share/info' '/usr/local/share/info/libffi.info'

 /bin/mkdir -p '/usr/local/lib/pkgconfig'

 /usr/bin/install -c -m  libffi.pc '/usr/local/lib/pkgconfig'

make[]: Leaving directory `/root/python/libffi-3.2./x86_64-unknown-linux-gnu'

make[]: Leaving directory `/root/python/libffi-3.2./x86_64-unknown-linux-gnu'

make[]: Leaving directory `/root/python/libffi-3.2./x86_64-unknown-linux-gnu'

source /etc/profile

scrapy runspider myspider.py