C0reFast记事本

to inspire confidence in somebody.

一条syslog信息包含三部分,PRI, HEADER和MSG,其中PRI是<>扩起来的一个数字,这个数字就代表了不同Facility和Severity的消息。
其中Facility, Severity的数字代号列表如下:

Facility:

Code Keyword Description
0 kern kernel messages
1 user user-level messages
2 mail mail system
3 daemon system daemons
4 auth security/authorization messages
5 syslog messages generated internally by syslogd
6 lpr line printer subsystem (archaic subsystem)
7 news network news subsystem (archaic subsystem)
8 uucp UUCP subsystem (archaic subsystem)
9 clock daemon
10 authpriv security/authorization messages
11 ftp FTP daemon
12 - NTP subsystem
13 - log audit
14 - log alert
15 cron scheduling daemon
16 local0 local use 0 (local0)
17 local1 local use 1 (local1)
18 local2 local use 2 (local2)
19 local3 local use 3 (local3)
20 local4 local use 4 (local4)
21 local5 local use 5 (local5)
22 local6 local use 6 (local6)
23 local7 local use 7 (local7)

Severity:

Code Keyword Description
0 emerg System is unusable
1 alert Should be corrected immediately
2 crit Critical conditions
3 err Error conditions
4 warning May indicate that an error will occur if action is not taken.
5 notice Events that are unusual, but not error conditions.
6 info Normal operational messages that require no action.
7 debug Information useful to developers for debugging the application.

针对PRI的计算公式:PRI = FacilityCode*8 + SeverityCode
举个例子: local3.info的日志,它的PRI就是19*8+6=158,所以这条消息在传输中的格式为<158> {HEADER} {MEG}
再一个例子,如果看到一条PRI为14的消息,那么它实际的级别就是user.info (1*8+6=14)

参考:

  1. https://tools.ietf.org/html/rfc3164
  2. https://wiki.archlinux.org/index.php/systemd#Journal

这几天翻了翻项目的代码,看到了一个非常简单的代理程序,使用poll实现,可以在代理过程中输出数据流,基本上算是教科书级别的poll使用例子了,分享一下:

#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iterator>
#include <iostream>
#include <unistd.h>
#include <errno.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <fcntl.h>
#include <poll.h>

#define PROXY_PORT  8888
#define BIN_TXT     0
#define RAW_TXT     1

static const char *backendHost = 0;
static int backendPort;
static int oflag = RAW_TXT;

bool proxy(int cfd);

// g++ -o proxy proxy.cpp -I. -Wall
int main(int argc, char *argv[])
{
    if (argc == 3 || argc == 4) {
        backendHost = argv[1];  
        backendPort = atoi(argv[2]);    
        if (argc == 4) oflag = atoi(argv[3]);
    } else {
        fprintf(stderr, "usage: %s BackendHost BackendPort oflag\n", argv[0]);  
        exit(-1);
    }

    int fd = socket(AF_INET, SOCK_STREAM, 0);

    struct sockaddr_in addr;
    memset(&addr, 0x00, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_port = htons(PROXY_PORT);
    addr.sin_addr.s_addr = htonl(INADDR_ANY);

    int flags = 1;
    setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (void *)&flags, sizeof(flags));

    if (bind(fd, (struct sockaddr*)&addr, sizeof(addr)) != 0) {
        fprintf(stderr, "[error] bind error, %s\n", strerror(errno));
        return EXIT_FAILURE;
    }

    listen(fd, 10);

    int cfd;
    while ((cfd = accept(fd, NULL, NULL)) > 0) {
#ifdef USE_FORK
        pid_t pid = fork();
        if (pid == 0) {
#endif
            proxy(cfd);
            close(cfd);
#ifdef USE_FORK
            exit(0);
        } else if (pid == -1) {
            fprintf(stderr, "[error] fork error, %s\n", strerror(errno));   
        } else {
            close(cfd); 
        }
#endif
    }
    close(fd);

    return EXIT_SUCCESS;
}

static int connectBackend()
{
    int sfd = socket(AF_INET, SOCK_STREAM, 0);
    if (sfd == -1) {
        fprintf(stderr, "socket error, %s\n", strerror(errno)); 
        return -1;
    }

    struct sockaddr_in addr;
    memset(&addr, 0x00, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_port = htons(backendPort);
    inet_pton(AF_INET, backendHost, &addr.sin_addr);

    if (connect(sfd, (struct sockaddr*)&addr, sizeof(addr)) != 0) {
        fprintf(stderr, "connect backed error, %s\n", strerror(errno));
        close(sfd);
        return -1;
    }

    fprintf(stdout, "[debug %d] connect backend ok\n", getpid());
    return sfd;
}

static void txtPrint(const char *buffer, size_t size)
{
    /* stderr unbuffered */
    if (oflag == BIN_TXT) {
        for (size_t i = 0; i < size; ++i) {
            fprintf(stderr, "%02x", (unsigned char) buffer[i]);
        }
    } else {
        for (size_t i = 0; i < size; ++i) {
            if (isprint((unsigned char) buffer[i])) {
                fprintf(stderr, "%c", buffer[i]);   
            } else {
                fprintf(stderr, "%%%02x", (unsigned char) buffer[i]);
            }
        }
    }
}

bool proxy(int cfd)
{
    const size_t len = 128;
    char buf[len];
    ssize_t n;

    fcntl(cfd, F_SETFL, fcntl(cfd, F_GETFL) | O_NONBLOCK);

    int sfd = connectBackend();
    if (sfd == -1) return false;

    fcntl(sfd, F_SETFL, fcntl(sfd, F_GETFL) | O_NONBLOCK);

    struct pollfd pfd[2] = {
        {cfd, POLLIN, 0},   
        {sfd, POLLIN, 0}
    };

    int nfds = 2;

    bool stop = false;
    while (!stop) {
        int nready = poll(pfd, nfds, -1);
        if (nready == -1) {
            fprintf(stderr, "[error] poll, %s\n", strerror(errno)); 
            close(sfd);
            return false;
        }

        for (int i = 0; i < nfds; ++i) {
            if (pfd[i].revents & POLLIN) {
                while ((n = recv(pfd[i].fd, buf, len, 0)) > 0) {
                    int fd = (pfd[i].fd == cfd) ? sfd : cfd;
                    ssize_t nn;
                    size_t offset = 0;
                    while ((nn = send(fd, buf + offset, n - offset, 0)) > 0) {
                        offset += nn;
                        if (offset == (size_t) n) break;
                    }
                    struct timeval tv;
                    gettimeofday(&tv, 0);
                    if (pfd[i].fd == cfd) {
                        fprintf(stdout, "[debug] %lu:%lu client read\n", (unsigned long) tv.tv_sec, (unsigned long) tv.tv_usec);
                    } else {
                        fprintf(stdout, "[debug] %lu:%lu server read\n", (unsigned long) tv.tv_sec, (unsigned long) tv.tv_usec);
                    }
                    txtPrint(buf, n);
                }
                if (n == 0) {
                    if (pfd[i].fd == cfd) {
                        fprintf(stdout, "[debug] client closed\n");
                    } else {
                        fprintf(stdout, "[debug] server closed\n");
                    }
                    stop = true;
                } else if (n == -1 && errno != EAGAIN) {
                    fprintf(stderr, "[error] read %s error, %s\n", pfd[i].fd == cfd ? "client" : "backend",
                                                                 strerror(errno));  
                    close(sfd);
                    return false;
                }
            }
        }
    }

    fprintf(stdout, "[debug %d] disconnect\n", getpid());

    close(sfd);
    return true;
}

使用g++ -o proxy proxy.cpp -I. -Wall 命令编译,运行时直接指定需要代理的后端IP和Port即可,也支持fork以支持多条链接。

Python中的上下文管理器可以允许精确地分配和释放资源,最常用的就是使用with语句,比如:

with open('/tmp/file_x', 'w') as file_x:
    file_x.write('Hello')

with结束,文件也会被安全的关闭。不用担心回收资源的问题了。

如果一个自定义的类也想支持类似的调用方式,需要实现__enter__(self)__exit__(self, type, value, traceback)这两个方法,具体的:

class File(object):
    def __init__(self, file_name, method):
        self.file_obj = open(file_name, method)
    def __enter__(self):
        return self.file_obj
    def __exit__(self, type, value, traceback):
        self.file_obj.close()

其中__enter__方法将打开的文件返回给with语句。
对于__exit__(self, type, value, traceback)方法,会在with语句退出时调用,如果在执行中发现异常,则异常的type,value和traceback会被传递给__exit__方法,在__exit__中可以对异常进行相应的处理,如果最终
__exit__方法返回None,则认为异常被正确处理了,如果返回的不是None,则这个异常会被with抛出,期待上层进行相应的处理。

除了上面的方法,Python还提供了一个contextlib模块,通过这个模块加上装饰器(decorators)和生成器(generators),也能实现类似的功能:

from contextlib import contextmanager

@contextmanager
def open_file(name):
    f = open(name, 'w')
    yield f
    f.close()

这样在使用中,open_file变成了一个生成器,所以contextmanager可以通过调用这个生成器next()方法控制资源的释放,具体的源代码在这里:

# 代码有所省略,具体可以参考: https://github.com/python/cpython/blob/master/Lib/contextlib.py
class _GeneratorContextManager(_GeneratorContextManagerBase,
                               AbstractContextManager,
                               ContextDecorator):
    """Helper for @contextmanager decorator."""

    def _recreate_cm(self):
        return self.__class__(self.func, self.args, self.kwds)

    def __enter__(self):
        try:
            return next(self.gen)
        except StopIteration:
            raise RuntimeError("generator didn't yield") from None

    def __exit__(self, type, value, traceback):
        if type is None:
            try:
                next(self.gen)
            except StopIteration:
                return False
            else:
                raise RuntimeError("generator didn't stop")
        else:
            if value is None:
                value = type()
            try:
                self.gen.throw(type, value, traceback)
            except StopIteration as exc:
                return exc is not value
            except RuntimeError as exc:
                if exc is value:
                    return False
                if type is StopIteration and exc.__cause__ is value:
                    return False
                raise
            except:
                if sys.exc_info()[1] is value:
                    return False
                raise
            raise RuntimeError("generator didn't stop after throw()")

def contextmanager(func):
    @wraps(func)
    def helper(*args, **kwds):
        return _GeneratorContextManager(func, args, kwds)
    return helper

参考:

  1. http://book.pythontips.com/en/latest/context_managers.html
  2. https://github.com/python/cpython/blob/master/Lib/contextlib.py

找到一个命令行版本的speedtest.net,可以在没有浏览器的情况下进行网络测速,具体的地址在[sivel/speedtest-cli][1]
[1]: https://github.com/sivel/speedtest-cli

单文件,只依赖Python,可以直接下载:

curl -Lo speedtest-cli https://raw.githubusercontent.com/sivel/speedtest-cli/master/speedtest.py
chmod +x speedtest-cli

下载后直接运行即可:

[root@test ~]# ./speedtest-cli
Retrieving speedtest.net configuration...
Testing from XXX Networks (x.x.x.x)...
Retrieving speedtest.net server list...
Selecting best server based on ping...
Hosted by Atlantic Metro (Los Angeles, CA) [1.30 km]: 1.565 ms
Testing download speed................................................................................
Download: 631.41 Mbit/s
Testing upload speed....................................................................................................
Upload: 48.73 Mbit/s

OpenStack Swift中,object replicator的作用是在系统遇到诸如临时的网络中断或磁盘故障后使系统处于一致状态。object replicator会将本地数据与每个远程副本进行比较,以确保它们都包含最新版本。下面会简单分析一下object replicator的代码,了解一下整个Replication的工作流程。

阅读全文 »

在我们内部的系统中,有一个tcp的代理服务,用户所有的网络相关的请求,比如访问外网,或者访问在内网的某些服务,都需要通过这个服务,一方面是实现对外网访问的计费,另外也通过白名单机制,对应用的内网访问进行相应的限制。
随着业务量的增加,发现提供服务的机器负载逐渐变高,当流量高峰的时候,经常出现客户端无法连接的情况,本来这个服务也是一个无状态的服务,可以很方便的水平扩容,在添加机器的同时,也尝试去分析一下程序本身的瓶颈,看能否提升一下程序本身的处理能力,通过分析和优化,还是在一定程度上提升了处理能力

阅读全文 »

在Linux Kernel 4.3中,引入了一个新的cgroups子系统pids,通过这个子系统,可以实现对某个控制组中进程和线程的总数进行限制。
使用前,首先需要挂载该子系统(对于很多的发行版,默认是会挂载的):

[root@test ~]# mkdir -p /sys/fs/cgroup/pids
[root@test ~]# mount -t cgroup -o pids none /sys/fs/cgroup/pids

首先创建一个新的控制组test_max_proc:

[root@test ~]# mkdir /sys/fs/cgroup/pids/test_max_proc
[root@test ~]# ls -l /sys/fs/cgroup/pids/test_max_proc/
total 0
-rw-r--r-- 1 root root 0 Apr 26 09:11 cgroup.clone_children
-rw-r--r-- 1 root root 0 Apr 26 09:11 cgroup.procs
-rw-r--r-- 1 root root 0 Apr 26 09:11 notify_on_release
-r--r--r-- 1 root root 0 Apr 26 09:11 pids.current
-r--r--r-- 1 root root 0 Apr 26 09:11 pids.events
-rw-r--r-- 1 root root 0 Apr 26 09:11 pids.max
-rw-r--r-- 1 root root 0 Apr 26 09:11 tasks

其中,pids.max控制该组中最多可以拥有的进程数,其中线程也包含在其中。pids.current存储了当前控制组的进程(线程)总数。cgroup.procs是需要限制的进程pid。

[root@test ~]# echo 2 > /sys/fs/cgroup/pids/test_max_proc/pids.max
[root@test ~]# echo $$ > /sys/fs/cgroup/pids/test_max_proc/cgroup.procs
[root@test ~]# cat /sys/fs/cgroup/pids/parent/pids.current
2
[root@test ~]# /bin/echo "Here's some processes for you." | cat
bash: fork: retry: Resource temporarily unavailable
bash: fork: retry: Resource temporarily unavailable
bash: fork: retry: Resource temporarily unavailable

可以看到限制生效了。由于Linux的线程也是类似进程的实现,因此,当程序有多个线程时,进程和线程的总数也不能超过设定的值

参考:

  1. http://man7.org/linux/man-pages/man7/cgroups.7.html
  2. https://www.kernel.org/doc/Documentation/cgroup-v1/pids.txt

本篇是Building a Consistent Hashing Ring的翻译,原文一步步描述了一个一致性哈希环的构建过程,对于OpenStack Swift存储,对应的Ring文件,其实就是一个一致性哈希环。
这篇文章讲述了OpenStack Swift Ring文件的构建原理。目前翻译了第一部分和第二部分,包含了最原始的算法,并最终引入虚拟节点,减少扩容时的数据移动

阅读全文 »

ArchLinux下网易云音乐会有偶然的白屏情况,是由于不支持某些emoji字体导致的,可以安装noto-fonts-emoji,然后配置一下字体即可解决:

sudo pacman -S noto-fonts-emoji

配置~/.config/fontconfig/conf.d/51-noto-color-emoji.conf文件:

<?xml version="1.0"?>
<!DOCTYPE fontconfig SYSTEM "fonts.dtd">
<fontconfig>
    <selectfont>
        <acceptfont>
            <pattern>
                <patelt name="family"><string>Noto Color Emoji</string></patelt>
            </pattern>
        </acceptfont>
    </selectfont>
    <match target="font">
        <test name="family">
            <string>Noto Color Emoji</string>
        </test>
        <edit name="scalable" mode="assign"><bool>true</bool></edit>
        <edit name="embeddedbitmap" mode="assign"><bool>true</bool></edit>
        <edit name="hinting" mode="assign"><bool>true</bool></edit>
        <edit name="hintstyle" mode="assign"><const>hintfull</const></edit>
    </match>
    <match target="pattern">
        <test name="family" qual="first" compare="contains">
            <string>emoji</string>
        </test>
        <edit mode="assign" name="color">
            <bool>true</bool>
        </edit>
        <edit mode="assign" name="family">
            <string>Noto Color Emoji</string>
        </edit>
    </match>
    <match target="pattern">
        <edit name="family" mode="prepend">
            <string>Noto Color Emoji</string>
        </edit>
    </match>
</fontconfig>

参考: https://blog.judge.moe/archives/90/

0%