用varnish来加速图片服务器

varnish是和squid类似的高性能开源HTTP加速器,我这里用来缓存图片,js,css等小文件

varnish cache 192.168.0.15 centos6.0
nagios www后端 192.168.0.11 centos5.3

1.安装varnish

wget http://repo.varnish-cache.org/source/varnish-3.0.0.tar.gz
tar zxvf varnish-3.0.0.tar.gz
cd varnish-3.0.0
./configure –prefix=/opt/varnish-3.0.0
make
make install
ln -s /opt/varnish-3.0.0 /opt/varnish

2.设置权限

cd /opt/varnish
#varnish以www:website来运行
chown -R www:website /opt/varnish/var/varnish/
mkdir /var/log/varnish
chown -R www:website /var/log/varnish
chown -R www:website /opt/varnish/var/varnish/`hostname`
mkdir /opt/varnish/var/varnish/`hostname`

3.配置文件

#查看默认配置文件
cat etc/varnish/default.vcl

#编辑新配置文件
vi etc/vcl.conf

#http请求处理过程
#1,receive请求入口状态，根据vcl判断pass还是lookup本地查询
#lookup，在hash表中查找数据，若找到则进入hit状态，否则进入fetch状态
#pass，选择后台，进入fetch状态
#fetch，对请求进行后端的获取，发送请求，获得数据，并进行本地存储
#deliver，将数据发送给客户端，进入done
#done,处理结束
backend wwwserver {
.host = “192.168.0.11”;
.port = “80”;
}
backend staticserver {
.host = “192.168.0.11”;
.port = “80”;
}
acl purge {
“localhost”;
“127.0.0.1”;
“192.168.1.0”/24;
}

sub vcl_recv {
if (req.request == “PURGE”) {
if (!client.ip ~ purge) {
error 405 “Not allowed.”;
}
return(lookup);
}
#去除cookie
if (req.request == “GET” && req.url ~ “^/[^?]+\.(jpeg|jpg|png|gif|ico|swf|js|css|txt|zip|html|htm)(\?.*|)$”) {
unset req.http.Cookie;
}
#判断req.http.X-Forwarded-For 如果前端有多重反向代理，这样可以获取客户端IP地址。
if (req.http.x-forwarded-for)
{
set req.http.X-Forwarded-For = req.http.X-Forwarded-For + “, “+ client.ip;
}
else {
set req.http.X-Forwarded-For = client.ip;
}

#浏览器Accept-Encoding兼容
if (req.http.Accept-Encoding) {
if (req.url ~ “\.(jpg|png|gif|jpeg)$”) {
remove req.http.Accept-Encoding;
} elsif (req.http.Accept-Encoding ~ “gzip”) {
set req.http.Accept-Encoding = “gzip”;
} elsif (req.http.Accept-Encoding ~ “deflate”) {
set req.http.Accept-Encoding = “deflate”;
} else {
remove req.http.Accept-Encoding;
}
}

if (req.http.host ~ “^blog.c1gstudio.com”) {
set req.backend = wwwserver;
if (req.request != “GET” && req.request != “HEAD”) {
return(pipe);
}
elseif(req.url ~ “\.(php|cgi)($|\?)”) {
return(pass);
}
else {

return(lookup);
}
} elsif (req.http.host ~ “^static.c1gstudio.net”) {
#第二个域名
set req.backend = staticserver;
}else {
error 404 “Cache Server”;
return(lookup);
}
}

sub vcl_hit {
if (req.request == “PURGE”) {
set obj.ttl = 0s;
error 200 “Purged.”;
}
}

sub vcl_miss {
if (req.request == “PURGE”) {
error 404 “Not in cache.”;
}
}

sub vcl_fetch {
if (req.request == “GET” && req.url ~ “\.(jpeg|jpg|png|gif|ico|swf|js|css|txt|zip)$”) {
set beresp.ttl = 600s;
unset beresp.http.set-cookie;
}
else {
set beresp.ttl = 3600s;
}
}
#显示是否命中
sub vcl_deliver{
if (obj.hits > 0) {
set resp.http.X-Cache = “Server-1-HIT”;
set resp.http.X-Cache-Hits = obj.hits;
} else {
set resp.http.X-Cache = “Server-1-MISS”;
}
unset resp.http.X-Varnish;
set resp.http.Via = “1.1 Xcache”;
}

4.启动varnish

/opt/varnish/sbin/varnishd -n /opt/varnish/var/varnish -f /opt/varnish/etc/vcl.conf -a 0.0.0.0:80 -s malloc,1G -g website -u www -T 127.0.0.1:3200 -p sess_workspace=64768 -p thread_pools=2 -p listen_depth=4096 -p first_byte_timeout=10 -p sess_timeout=15 -w 200,5000,10

#参数说明

-n vcache / #临时文件实例名.如果以”/”开头,就必须是一个可用的路径.
-a :80 / #服务所在端口.”:80″是默认所有网络都建立80端口,”:”前面是服务器IP.
-T :5000 / #管理端口.
-s file,/data1/vcache,80g / #虚拟内存文件映射类型,路径以及容量. 包括两种类型”malloc”和”file”
-s file,/data2/vcache,80g / #malloc是内存+swap交换模式.很简单.没得说.
-s file,/data3/vcache,80g / #file是mmap的文件内存映射机制.(具体情况,参阅”mmap”函数说明)
-s file,/data4/vcache,80g /
-f /usr/local/varnish/etc/varnish.vcl / #VCL文件路径.
-P /var/run/varnish.pid / #PID文件地址.
-w 200,5000,10 / #工作进程数.三个参数分别是:,,
-h classic,16383 / #hash列表类型,以及长度.默认长度是16383.具体用处和调整实际效果要等我看完源代码才知道.
-p user=www / #”-p”是变量配置参数
-p group=website/ #服务运行用户和用户组配置.
-p thread_pools=4 / #进程connections pools的个数,数量越多,越耗用cpu和mem,但是处理并发能力越强.
#系统手册上说,一个cpu用一个.
-p listen_depth=4096 / #TCP队列长度.默认是1024.
-p first_byte_timeout=10 #从后端接受第一个字节的超时时间。默认60秒
-p between_bytes_timeout=60 #从后端接收数据后,连接空闲时间,默认60秒
-p sess_timeout=15 #客户端和varnish连接超时时间,默认5秒

===============2015-1-16更新===============
4.2不关闭varnish,重新载入配置

telnet 127.0.0.1 3200
使用命令
vcl.load new.vcl /opt/varnish/etc/vcl.conf
(编译出错的话会有提示，成功会返回200)
200
然后使用
vcl.use new.vcl
(成功后同样会返回200)
200

crtl+]
quit
=============================================

5.记录日志
/opt/varnish/bin/varnishncsa -n /opt/varnish/var/varnish -w /var/log/varnish/varnish.log &

#定时切割日志
vi /opt/shell/cutvarnishlog.sh

#!/bin/sh
# 0 0 * * * /bin/sh /opt/shell/cutvarnishlog.sh > /dev/null 2>&1
date=$(date -d “yesterday” +”%Y%m%d”)
pkill -9 varnishncsa
mv /var/log/varnish/varnish.log /var/log/varnish/varnish.${date}.log
/opt/varnish/bin/varnishncsa -n /opt/varnish/var/varnish -w /var/log/varnish/varnish.log &
mkdir -p /var/log/varnish/old
gzip -c /var/log/varnish/varnish.${date}.log > /var/log/varnish/old/varnish.${date}.log.gz
rm -f /var/log/varnish/varnish.${date}.log
rm -f /var/log/varnish/old/varnish$(date -d “-1 month” +”%Y%m*”).log.gz

crontab -e

0 0 * * * /bin/sh /opt/shell/cutvarnishlog.sh > /dev/null 2>&1

6.查看运行统计
/opt/varnish/bin/varnishstat -n /opt/varnish/var/varnish

1+01:13:37 /opt/varnish/var/varnish
Hitrate ratio: 10 100 288
Hitrate avg: 0.9987 0.9981 0.9978

22251295 371.40 245.01 client_conn – Client connections accepted
22250487 371.40 245.00 client_req – Client requests received
22185321 371.40 244.29 cache_hit – Cache hits
62904 0.00 0.69 cache_miss – Cache misses
4615 0.00 0.05 backend_conn – Backend conn. success
22 0.00 0.00 backend_fail – Backend conn. failures
59164 0.00 0.65 backend_reuse – Backend conn. reuses
456 0.00 0.01 backend_toolate – Backend conn. was closed
59622 0.00 0.66 backend_recycle – Backend conn. recycles
47470 0.00 0.52 fetch_length – Fetch with Length
16307 0.00 0.18 fetch_chunked – Fetch chunked
2 0.00 0.00 fetch_close – Fetch wanted close
1873 . . n_sess_mem – N struct sess_mem
1834 . . n_sess – N struct sess
655 . . n_object – N struct object
685 . . n_objectcore – N struct objectcore
784 . . n_objecthead – N struct objecthead
405 . . n_waitinglist – N struct waitinglist
2 . . n_vbc – N struct vbc
31 . . n_wrk – N worker threads
381 0.00 0.00 n_wrk_create – N worker threads created
2584 0.00 0.03 n_wrk_queued – N queued work requests
2 . . n_backend – N backends
62227 . . n_expired – N expired objects
5365503 . . n_lru_moved – N LRU moved objects
1362 0.00 0.01 losthdr – HTTP header overflows
18551363 326.47 204.27 n_objwrite – Objects sent with write
22251295 371.40 245.01 s_sess – Total Sessions
22250487 371.40 245.00 s_req – Total Requests
898 0.00 0.01 s_pass – Total pass
63779 0.00 0.70 s_fetch – Total fetch
7539848276 127352.96 83022.43 s_hdrbytes – Total header bytes
141933911830 2248780.45 1562856.20 s_bodybytes – Total body bytes
22251292 371.40 245.01 sess_closed – Session Closed
1 0.00 0.00 sess_herd – Session herd
998035729 16610.26 10989.53 shm_records – SHM records
89193699 1488.60 982.13 shm_writes – SHM writes
328009 8.99 3.61 shm_cont – SHM MTX contention
385 0.00 0.00 shm_cycles – SHM cycles through buffer
1387 0.00 0.02 sms_nreq – SMS allocator requests

7.管理清除缓存
7.1通过Varnish管理端口进行管理
/opt/varnish/bin/varnishadm -T 127.0.0.1:3200 help

CLI connected to 127.0.0.1:3200
help [command]
ping [timestamp]
auth response
quit
banner
status
start
stop
vcl.load
vcl.inline
vcl.use
vcl.discard
vcl.list
vcl.show
param.show [-l] []
param.set
panic.show
panic.clear
storage.list
ban.url
ban [&& ]…
ban.list

通过Varnish管理端口清除缓存,支持正则表达式,1.0时为url.purge参数：
/opt/varnish/bin/varnishadm -T 127.0.0.1:3200 ban.url /shanghai-4.html

例：清除所有缓存：
/opt/varnish/bin/varnishadm -T 127.0.0.1:3200 ban.url *$

7.2通过telnet方式清除

telnet 127.0.0.1 3200
Trying 127.0.0.1 …
Connected to 127.0.0.1.
Escape character is ‘^]’.
200 205
—————————–
Varnish Cache CLI 1.0
—————————–
Linux,2.6.32-71.el6.i686,i686,-smalloc,-smalloc,-hcritbit

Type ‘help’ for command list.
Type ‘quit’ to close CLI session.

help
200 401
help [command]
ping [timestamp]
auth response
quit
banner
status
start
stop
vcl.load
vcl.inline
vcl.use
vcl.discard
vcl.list
vcl.show
param.show [-l] []
param.set
panic.show
panic.clear
storage.list
ban.url
ban [&& ]…
ban.list

#1.0时的方法现在不支持
purge.url /shanghai-4.html
200 0 101 44
Unknown request.
Type ‘help’ for more info.

#正确方法
ban.url /shanghai-4.html
200 0

7.3通过php等其它web请求清除缓存

function purge($ip,$port=80,$domain, $url)
{
$errstr = ”;
$errno = ”;
$fp = fsockopen ($ip, $port, $errno, $errstr, 2);
if (!$fp)
{
return false;
}
else
{
$out = “PURGE $url HTTP/1.1\r\n”;
$out .= “Host:$domain\r\n”;
$out .= “Connection: close\r\n\r\n”;
fputs ($fp, $out);
$out = fgets($fp , 4096);
fclose ($fp);
return true;
}
}
purge(‘192.168.0.15′,’80’,’blog.c1gstudio.com’,’/shanghai-4.html’);

8.varnish的nginx前端
测试下来nginx和varnish在同一机器上会产生大量time_wait,单独使用没有问题.

upstream mysvr {
server 127.0.0.1:82;
}

server
{
listen 80;
server_name static.c1gstudio.net;
index index.html index.htm index.php;
root /opt/lampp/htdocs/web;

location ~/\.ht {
deny all;
}
location ~(favicon.ico) {
log_not_found off;
expires 99d;
break;
}
location ~ .*\.(php|html|htm)?$
{
return 403;
}

location / {
valid_referers none blocked *.c1gstudio.com *.c1gstudio.net ;
if ($invalid_referer) {
rewrite ^/ http://leech.c1gstudio.com/leech.gif;
return 412;
break;
}

proxy_pass http://mysvr;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}

access_log /var/log/nginx/static.c1gstudio.net.log access;
}

9.内核优化
vi /etc/sysctl.conf

net.ipv4.tcp_fin_timeout = 30
net.ipv4.tcp_keepalive_time = 300
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.ip_local_port_range = 5000 65000

sysctl -p

varnish服务器运行基本没有负载

top – 15:54:34 up 34 days, 23:49, 1 user, load average: 0.00, 0.01, 0.00
Tasks: 125 total, 1 running, 124 sleeping, 0 stopped, 0 zombie
Cpu(s): 1.8%us, 1.3%sy, 0.0%ni, 95.0%id, 0.4%wa, 0.0%hi, 1.5%si, 0.0%st
Mem: 2070548k total, 2017996k used, 52552k free, 83556k buffers
Swap: 2097144k total, 0k used, 2097144k free, 1612756k cached

PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
26631 www 20 0 228m 134m 81m S 7.6 6.7 74:46.86 varnishd
6070 www 20 0 31852 25m 1000 S 3.3 1.3 7:28.79 nginx
6071 www 20 0 31076 24m 1000 S 2.0 1.2 7:22.34 nginx
6068 www 20 0 31356 25m 976 S 1.7 1.3 7:21.36 nginx

tcp状态
netstat -n | awk ‘/^tcp/ {++S[$NF]} END {for(a in S) print a, S[a]}’

LAST_ACK 9
SYN_RECV 5
CLOSE_WAIT 3
ESTABLISHED 2083
FIN_WAIT1 95
FIN_WAIT2 247
TIME_WAIT 14412

2011-11-17更新
2011-11-15更新

参考:
Varnish-2.1.2 安装与配置.pdf
varnish浅析.pdf
varnish文件缓存实现2008-11-22.pdf
三个文件包下载varnishdocs 736k

http://blog.s135.com/post/313/
http://eneplace.com/2011/01/varnish-cookies-querystrings.html
https://www.varnish-cache.org/docs/3.0/reference/vcl.html#variables