bugcy013
7/12/2014 - 9:26 PM
pmacct installation with ubuntu 14.04
pmacct installation with ubuntu 14.04
Enable RabbitMQ application repository:
echo "deb http://www.rabbitmq.com/debian/ testing main" >> /etc/apt/sources.list
Add the verification key for the package:
curl http://www.rabbitmq.com/rabbitmq-signing-key-public.asc | sudo apt-key add -
Update the sources with our new addition from above:
apt-get update
And finally, download and install RabbitMQ:
sudo apt-get install rabbitmq-server
In order to manage the maximum amount of connections upon launch, open up and edit the following configuration file using nano:
sudo nano /etc/default/rabbitmq-server
Uncomment the limit line (i.e. remove #) before saving and exit by pressing CTRL+X followed with Y.
To enable RabbitMQ Management Console, run the following:
sudo rabbitmq-plugins enable rabbitmq_management
Once you've enabled the console, it can be accessed using your favourite web browser by visiting: http://[your droplet's IP]:15672/.
The default username and password are both set “guest” for the log in.
Note: If you enable this console after running the service, you will need to restart it for the changes to come into effect. See the relevant management section below for your operating system to be able to do it.
Download jansson
http://www.digip.org/jansson/releases/jansson-2.6.tar.gz
untar and ./configure && make && make install
RabbitMQ C AMQP client library
Download
https://github.com/alanxz/rabbitmq-c/releases/download/v0.5.0/rabbitmq-c-0.5.0.tar.gz
untar and ./configure && make && make install
for GeoIP Support you need to compile geoip-aapi-c
https://github.com/maxmind/geoip-api-c/releases/download/v1.6.2/GeoIP-1.6.2.tar.gz
untar and ./configure && make && make install
./configure --enable-rabbitmq --with-rabbitmq-libs=/usr/local/lib --with-rabbitmq-includes=/usr/local/include/ --enable-jansson --enable-geoip
root@HP-ProBook-4430s:~/Downloads/pmacct-1.5.0rc3# ./configure --enable-rabbitmq --with-rabbitmq-libs=/usr/local/lib --with-rabbitmq-includes=/usr/local/include/ --enable-jansson
loading cache ./config.cache
checking for a BSD compatible install... (cached) /usr/bin/install -c
checking whether build environment is sane... yes
checking whether make sets ${MAKE}... (cached) yes
checking for working aclocal-1.4... missing
checking for working autoconf... found
checking for working automake-1.4... missing
checking for working autoheader... found
checking for working makeinfo... missing
checking for gcc... (cached) gcc
checking whether the C compiler (gcc ) works... yes
checking whether the C compiler (gcc ) is a cross-compiler... no
checking whether we are using GNU C... (cached) yes
checking whether gcc accepts -g... (cached) yes
checking OS... Linux
checking hardware... x86_64
checking for ranlib... (cached) ranlib
checking whether to enable debugging compiler options... no
checking whether to relax compiler optimizations... no
checking whether to disable shared objects... no
checking for dlopen... (cached) no
checking for dlopen in -ldl... (cached) yes
checking for gmake... (cached) make
checking whether make sets ${MAKE}... (cached) yes
checking for __progname... yes
checking for extra flags needed to export symbols... --export-dynamic
checking for static inline... yes
checking endianess... little
checking unaligned accesses... ok
checking whether to enable L2 features... yes
checking whether to enable IPv6 code... no
checking whether to enable IP prefix labels... checking default locations for pcap.h... found in /usr/include
checking default locations for libpcap... no
checking for pcap_dispatch in -lpcap... (cached) yes
checking for pcap_setnonblock in -lpcap... (cached) yes
checking packet capture type... linux
checking whether to enable MySQL support... checking how to run the C preprocessor... (cached) gcc -E
no
checking whether to enable PostgreSQL support... no
checking whether to enable MongoDB support... no
checking whether to enable SQLite3 support... no
checking whether to enable RabbitMQ/AMQP support... yes
checking your own RabbitMQ library... ok
checking your own RabbitMQ headers... ok
checking whether to enable GeoIP support... no
checking whether to enable Jansson support... yes
checking default locations for Jansson library... found in /usr/local/lib
checking default locations for jansson.h... found in /usr/local/include
checking for ANSI C header files... (cached) yes
checking for sys/wait.h that is POSIX.1 compatible... (cached) yes
checking for getopt.h... (cached) yes
checking for sys/select.h... (cached) yes
checking for sys/time.h... (cached) yes
checking for u_int64_t in sys/types.h... yes
checking for u_int32_t in sys/types.h... yes
checking for u_int16_t in sys/types.h... yes
checking for u_int8_t in sys/types.h... yes
checking for uint64_t in sys/types.h... no
checking for uint32_t in sys/types.h... no
checking for uint16_t in sys/types.h... no
checking for uint8_t in sys/types.h... no
checking whether to enable 64bit counters... yes
checking whether to enable multithreading in pmacct... yes
checking whether to enable ULOG support... no
checking return type of signal handlers... (cached) void
checking for strlcpy... (cached) no
checking for vsnprintf... (cached) yes
checking for setproctitle... (cached) no
checking for mallopt... (cached) yes
PLATFORM ..... : x86_64
OS ........... : Linux 3.13.0-24-generic (HP-ProBook-4430s)
COMPILER ..... : gcc
CFLAGS ....... : -O2 -g -O2 -I/usr/local/include -I/usr/local/include
LIBS ......... : -lpcap -ldl -L/usr/local/lib -lrabbitmq -L/usr/local/lib -ljansson -lpthread
SERVER_LIBS ...: -lnfprobe_plugin -Lnfprobe_plugin/ -lsfprobe_plugin -Lsfprobe_plugin/ -lbgp -Lbgp/ -ltee_plugin -Ltee_plugin/ -lisis -Lisis/
LDFLAGS ...... : -Wl,--export-dynamic
Now type 'make' to compile the source code.
Are you willing to get in touch with other pmacct users?
Join the pmacct mailing-list by sending a message to pmacct-discussion-subscribe@pmacct.net
Need for documentation and examples?
Read the README file or go to http://wiki.pmacct.net/
creating ./config.status
creating Makefile
creating src/Makefile
creating src/nfprobe_plugin/Makefile
creating src/sfprobe_plugin/Makefile
creating src/bgp/Makefile
creating src/tee_plugin/Makefile
creating src/isis/Makefile
root@HP-ProBook-4430s:~/Downloads/pmacct-1.5.0rc3#
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
Sample configuration writing data to falt_file
[/etc]$ cat nfacctd_dhana.conf
nfacctd_ip: 192.168.70.54
nfacctd_port: 9998
plugins: print[forensics]
aggregate[forensics]: src_host, dst_host, peer_src_ip, peer_dst_ip, in_iface, out_iface, timestamp_start, timestamp_end, src_port, dst_port, proto, tos, src_mask, dst_mask, src_as, dst_as, tcpflags
aggregate[int_traffic_matrix]: in_iface, peer_src_ip, peer_dst_ip, peer_dst_as
!plugins: print
!aggregate[inbound]: dst_host
!aggregate[outbound]: src_host
debug: true
!daemonize: true
pidfile: /var/run/nfacctd.pid
!print_refresh_time: 900
print_refresh_time: 90
print_history: 15m
print_output: json
print_output_file: /tmp/file-%Y%m%d-%H%M.txt
print_history_roundoff: m
for Execution
=============
nfacctd -f nfacctd_dhana.conf
ERROR ( 5m_ipip/amqp ): Connection failed to RabbitMQ: login
^CERROR ( 5m_ipip/amqp ): Connection failed to RabbitMQ: login
OK: Exiting ...
[~]$ cat /etc/nfacctd_amqp.conf
nfacctd_ip: 192.168.70.54
nfacctd_port: 9998
plugin_pipe_size: 32576000
plugin_buffer_size: 325760
debug: true
daemonize: false
nfacctd_disable_checks: true
nfacctd_time_new: true
! AMQP connection details
amqp_host: localhost
amqp_user: guest
amqp_passwd: guest
amqp_exchange: pmacct
amqp_routing_key: acct
plugins: amqp[5m_ipip]
! 5 minutely IP to IP
aggregate[5m_ipip]: src_host, dst_host, src_port, dst_port, proto, peer_src_ip
amqp_routing_key[5m_ipip]: 5m_ipip
amqp_history[5m_ipip]: 5m
amqp_time_roundoff[5m_ipip]: m
amqp_refresh_time[5m_ipip]: 300
[~]$
ERROR ( 5m_ipip/amqp ): We are missing data.
If you see this message once in a while, discard it. Otherwise some solutions follow:
- increase shared memory size, 'plugin_pipe_size'; now: '4096000'.
- increase buffer size, 'plugin_buffer_size'; now: '888'.
- increase system maximum socket size.
ERROR ( 5m_ipip/amqp ): We are missing data.
If you see this message once in a while, discard it. Otherwise some solutions follow:
- increase shared memory size, 'plugin_pipe_size'; now: '4096000'.
- increase buffer size, 'plugin_buffer_size'; now: '888'.
- increase system maximum socket size.
LINKS ::
========
http://www.menog.org/presentations/menog-13/203-Lucente_collecting_netflow_with_pmacct_v1.1.pdf
https://github.com/Tilka/pmacct/blob/master/CONFIG-KEYS
KEY: [ sql_host | mongo_host | amqp_host ]
DESC: defines the backend server IP/hostname (default: localhost).
KEY: [ sql_user | mongo_user | amqp_user ]
DESC: defines the username to use when connecting to the server. In MongoDB, if both
mongo_user and mongo_passwd directives are omitted, authentication is disabled;
if only one of the two is specified, the other is set to its default value.
(default: pmacct).
KEY: [ sql_passwd | mongo_passwd | amqp_passwd ]
DESC: defines the password to use when connecting to the server.In MongoDB, if both
mongo_user and mongo_passwd directives are omitted, authentication is disabled;
if only one of the two is specified, the other is set to its default value.
(default: arealsmartpwd).
KEY: [ sql_refresh_time | print_refresh_time | mongo_refresh_time | amqp_refresh_time ] (-r)
DESC: time interval, in seconds, between consecutive executions of the plugin cache scanner. The
scanner purges data into the plugin backend. Note: internally all these config directives
write to the same variable; when using multiple plugins it is recommended to bind refresh
time definitions to specific plugins, ie.:
plugins: mysql[x], mongodb[y]
sql_refresh_time[x]: 900
mongo_refresh_time[y]: 300
As doing otherwise can originate unexpected behaviours.
KEY: [ sql_history | print_history | mongo_history | amqp_history ]
VALUES: #[m|h|d|w|M]
DESC: enables historical accounting by placing accounted data into configurable time-bins. It
will use the 'stamp_inserted' (base time of the time-bin) and 'stamp_updated' (last time
the time-bin was touched) fields. The supplied value defines the time slot length during
which counters are accumulated. For a nice effect, it's adviceable to pair this directive
with 'sql_history_roundoff'. In nfacctd, where a flow can span across multiple time-bins,
flow counters are pro-rated (seconds timestamp resolution) over involved time-bins.
Note that this value is fully disjoint from the 'sql_refresh_time' directive which sets
the time intervals at which data has to be written to the RDBMS instead. The final effect
is close to time slots in a RRD file. Examples of valid values are: '5m' - five minutes,
'1h' - one hour, '4h' - four hours, '1d' - one day, '1w' - one week, '1M' - one month).
KEY: [ sql_history_offset | print_history_offset | mongo_history_offset | amqp_history_offset ]
DESC: Sets an offset to timeslots basetime. If history is set to 30 mins (by default creating
10:00, 10:30, 11:00, etc. time-bins), with an offset of 900 seconds (so 15 mins) it will
create 10:15, 10:45, 11:15, etc. time-bins. It expects a positive value, in seconds.
(default: 0)
KEY: [ sql_history_roundoff | print_history_roundoff | mongo_history_roundoff |
amqp_history_roundoff ]
VALUES [m,h,d,w,M]
DESC: enables alignment of minutes (m), hours (h), days of month (d), weeks (w) and months (M)
in print (to print_refresh_time) and SQL plugins (to sql_history and sql_refresh_time).
Suppose you go with 'sql_history: 1h', 'sql_history_roundoff: m' and it's 6:34pm. Rounding
off minutes gives you an hourly timeslot (1h) starting at 6:00pm; so, subsequent ones will
start at 7:00pm, 8:00pm, etc. Now, you go with 'sql_history: 5m', 'sql_history_roundoff: m'
and it's 6:37pm. Rounding off minutes will result in a first slot starting at 6:35pm; next
slot will start at 6:40pm, and then every 5 minutes (6:45pm ... 7:00pm, etc.). 'w' and 'd'
are mutually exclusive, that is: you can either reset the date to last Monday or reset the
date to the first day of the month.
KEY: [ sql_cache_entries | print_cache_entries | mongo_cache_entries | amqp_cache_entries ]
DESC: SQL and other plugins sport a Plugin Memory Cache (PMC) meant to accumulate bytes/packets
counters until next purging event (for further insights take a look to 'sql_refresh_time').
This directive sets the number of PMC buckets. Default value is suitable for most common
scenarios, however when facing large-scale networks, it's higly recommended to carefully
tune this parameter to improve performances. Use a prime number of buckets.
(default: sql_cache_entries: 32771, print_cache_entries: 16411)
KEY: amqp_exchange
DESC: Name of the AMQP exchange to publish data (default: pmacct).
KEY: amqp_exchange_type
DESC: Type of the AMQP exchange to publish data. Currently only 'direct' and 'fanout' types are
supported. (default: direct).
KEY: amqp_routing_key
DESC: Name of the AMQP routing key to attach to published data. Dynamic names are supported through
the use of variables, which are computed at the moment when data is purged to the backend. The
list of supported variables follows (default: acct):
$peer_src_ip Value of the peer_src_ip primitive of the record being processed.
$pre_tag Value of the tag primitive of the record being processed.
$post_tag Configured value of post_tag.
KEY: amqp_persistent_msg
VALUES: [ true | false ]
DESC: Marks messages as persistent so that a queue content does not get lost if RabbitMQ restarts.
Note from RabbitMQ docs: "Marking messages as persistent doesn't fully guarantee that a
message won't be lost. Although it tells RabbitMQ to save message to the disk, there is
still a short time window when RabbitMQ has accepted a message and hasn't saved it yet.
Also, RabbitMQ doesn't do fsync(2) for every message -- it may be just saved to cache and
not really written to the disk. The persistence guarantees aren't strong, but it is more
than enough for our simple task queue.".
rabbitmqctl add_vhost statuscheckvhost
rabbitmqctl add_user heartbeat alive
rabbitmqctl set_permissions -p statuscheckvhost heartbeat ".*" ".*" ".*"
rabbitmqctl set_user_tags heartbeat management
curl -i -u heartbeat:alive http://127.0.0.1:55672/api/aliveness-test/statuscheckvhost
HTTP/1.1 200 OK
Server: MochiWeb/1.1 WebMachine/1.9.0 (someone had painted it blue)
Date: Thu, 21 Feb 2013 22:20:10 GMT
Content-Type: application/json
Content-Length: 15
Cache-Control: no-cache
{"status":"ok"}
Logstash configuration File:
[/usr/local/logstash]$ cat conf/shipper_amqp.conf
input {
rabbitmq {
host => "localhost"
exchange => "pmacct"
key => "5m_ipip"
user => "guest"
password => "guest"
}
}
output {
stdout { codec => rubydebug }
}
[~/pmacct-1.5.0rc3/examples/amqp]$ cat amqp_receiver.py
#!/usr/bin/python
#
# If missing 'pika' read how to download it at:
# http://www.rabbitmq.com/tutorials/tutorial-one-python.html
#
import pika
connection = pika.BlockingConnection(pika.ConnectionParameters(
host='localhost'))
channel = connection.channel()
channel.exchange_declare(exchange='pmacct', type='direct')
channel.queue_declare(queue='acct_1')
channel.queue_bind(exchange='pmacct', routing_key='5m_ipip', queue='acct_1')
print ' [*] Example inspired from: http://www.rabbitmq.com/getstarted.html'
print ' [*] Waiting for messages on E=pmacct,direct RK=5m_ipip Q=acct_1 H=localhost. Edit code to change any parameter. To exit press CTRL+C'
def callback(ch, method, properties, body):
print " [x] Received %r" % (body,)
channel.basic_consume(callback,
queue='acct_1',
no_ack=True)
channel.start_consuming()
[~/pmacct-1.5.0rc3/examples/amqp]$
ASA(config)# flow-export destination Systems 192.168.70.54 9998
ASA(config)# access-list flow_export_acl permit ip any any
ASA(config)# class-map flow_export_class
ASA(config-cmap)# match access-list flow_export_acl
ASA(config)# policy-map global_policy
ASA(config-pmap)# class flow_export_class
ASA(config-pmap-c)# flow-export event-type all destination 192.168.70.54
flow-export delay flow-create
root@HP-ProBook-4430s:~# cat /etc/pmacctd_print.conf
debug: true
daemonize: false
interface: wlan0
! AMQP connection details
amqp_host: localhost
amqp_user: guest
amqp_passwd: guest
amqp_exchange: pmacct
!amqp_routing_key: acct
aggregate[inbound]: tag,src_host, dst_host, src_port, dst_port, proto
aggregate_filter[inbound]: dst net 192.168.0.0/24
aggregate[outbound]: tag,src_host, dst_host, src_port, dst_port, proto
aggregate_filter[outbound]: src net 192.168.0.0/24
!networks_file: /etc/pmacct/hosts
pre_tag_map: /etc/pmacct/pretag.map
plugins: amqp[inbound],amqp[outbound]
root@HP-ProBook-4430s:~# pmacctd -f /etc/pmacctd_print.conf
root@HP-ProBook-4430s:~# cat /etc/pmacct/hosts
192.168.0.0/24
root@HP-ProBook-4430s:~# cat /etc/pmacct/pretag.map
id=1 filter='dst net 192.168.0.0/16'
id=2 filter='src net 192.168.0.0/16'
root@HP-ProBook-4430s:~#