Kernel sysctl configuration file for Linux

# Version 1.9 - 2011-03-23
# Michiel Klaver - IT Professional
# http://klaver.it/linux/ for the latest version - http://klaver.it/bsd/ for a BSD variant
#
# This file should be saved as /etc/sysctl.conf and can be activated using the command:
# sysctl -e -p /etc/sysctl.conf
#
# For binary values, 0 is disabled, 1 is enabled.  See sysctl(8) and sysctl.conf(5) for more details.
#
# Tested with: Debian 4 etchnhalf kernel version 2.6.24 default stock out-of-the-box
#              Debian 5 kernel version 2.6.26 default stock out-of-the-box
#              CentOS 5.4 kernel 2.6.18 default stock out-of-the-box

#
# Intended use for dedicated server systems at high-speed networks with loads of RAM and bandwidth available
# Optimised and tuned for high-performance web/ftp/mail/dns servers with high connection-rates
# DO NOT USE at busy networks or xDSL/Cable connections where packetloss can be expected
# ----------

# Credits:
# http://www.enigma.id.au/linux_tuning.txt
# http://www.securityfocus.com/infocus/1729
# http://fasterdata.es.net/TCP-tuning/linux.html
# http://fedorahosted.org/ktune/browser/sysctl.ktune
# http://www.cymru.com/Documents/ip-stack-tuning.html
# http://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt
# http://www.frozentux.net/ipsysctl-tutorial/chunkyhtml/index.html
# http://knol.google.com/k/linux-performance-tuning-and-measurement
# http://www.cyberciti.biz/faq/linux-kernel-tuning-virtual-memory-subsystem/
# http://www.redbooks.ibm.com/abstracts/REDP4285.html
# http://www.speedguide.net/read_articles.php?id=121
# http://lartc.org/howto/lartc.kernel.obscure.html
# http://en.wikipedia.org/wiki/Sysctl



###
### GENERAL SYSTEM SECURITY OPTIONS ###
###

# Auto-reboot linux 30 seconds after a kernel panic
kernel.panic = 30
kernel.panic_on_oops = 30

# Controls the System Request debugging functionality of the kernel
kernel.sysrq = 0

# Controls whether core dumps will append the PID to the core filename.
# Useful for debugging multi-threaded applications.
kernel.core_uses_pid = 1

#Allow for more PIDs
kernel.pid_max = 65536

# The contents of /proc/<pid>/maps and smaps files are only visible to
# readers that are allowed to ptrace() the process
kernel.maps_protect = 1

#Enable ExecShield protection
kernel.exec-shield = 1
kernel.randomize_va_space = 1

# Controls the maximum size of a message, in bytes
kernel.msgmnb = 65536

# Controls the default maxmimum size of a mesage queue
kernel.msgmax = 65536



###
### IMPROVE SYSTEM MEMORY MANAGEMENT ###
###

# Increase size of file handles and inode cache
fs.file-max = 209708

# Do less swapping
vm.swappiness = 10
vm.dirty_ratio = 60
vm.dirty_background_ratio = 2

# specifies the minimum virtual address that a process is allowed to mmap
vm.mmap_min_addr = 4096

# No overcommitment of available memory
vm.overcommit_ratio = 0
vm.overcommit_memory = 0

# Set maximum amount of memory allocated to shm to 256MB
kernel.shmmax = 268435456
kernel.shmall = 268435456

# Keep at least 64MB of free RAM space available
vm.min_free_kbytes = 65536



###
### GENERAL NETWORK SECURITY OPTIONS ###
###

#Prevent SYN attack, enable SYNcookies (they will kick-in when the max_syn_backlog reached)
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_syn_retries = 5
net.ipv4.tcp_synack_retries = 2
net.ipv4.tcp_max_syn_backlog = 4096

# Disables packet forwarding
net.ipv4.ip_forward = 0
net.ipv4.conf.all.forwarding = 0
net.ipv4.conf.default.forwarding = 0
net.ipv6.conf.all.forwarding = 0
net.ipv6.conf.default.forwarding = 0

# Disables IP source routing
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.accept_source_route = 0
net.ipv6.conf.all.accept_source_route = 0
net.ipv6.conf.default.accept_source_route = 0

# Enable IP spoofing protection, turn on source route verification
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1

# Disable ICMP Redirect Acceptance
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv6.conf.all.accept_redirects = 0
net.ipv6.conf.default.accept_redirects = 0

# Disable Log Spoofed Packets, Source Routed Packets, Redirect Packets
net.ipv4.conf.all.log_martians = 0
net.ipv4.conf.default.log_martians = 0

# Decrease the time default value for tcp_fin_timeout connection
net.ipv4.tcp_fin_timeout = 15

# Decrease the time default value for connections to keep alive
net.ipv4.tcp_keepalive_time = 300
net.ipv4.tcp_keepalive_probes = 5
net.ipv4.tcp_keepalive_intvl = 15

# Don't relay bootp
net.ipv4.conf.all.bootp_relay = 0

# Don't proxy arp for anyone
net.ipv4.conf.all.proxy_arp = 0

# Turn on SACK
net.ipv4.tcp_dsack = 1
net.ipv4.tcp_sack = 1
net.ipv4.tcp_fack = 1

# Turn on the tcp_timestamps
net.ipv4.tcp_timestamps = 1

# Don't ignore directed pings
net.ipv4.icmp_echo_ignore_all = 0

# Enable ignoring broadcasts request
net.ipv4.icmp_echo_ignore_broadcasts = 1

# Enable bad error message Protection
net.ipv4.icmp_ignore_bogus_error_responses = 1

# Allowed local port range
net.ipv4.ip_local_port_range = 16384 65536

# Enable a fix for RFC1337 - time-wait assassination hazards in TCP
net.ipv4.tcp_rfc1337 = 1



###
### TUNING NETWORK PERFORMANCE ###
###

# Do a 'modprobe tcp_cubic' first
net.ipv4.tcp_congestion_control = cubic

# Turn on the tcp_window_scaling
net.ipv4.tcp_window_scaling = 1

# Increase the maximum total buffer-space allocatable
# This is measured in units of pages (4096 bytes)
net.ipv4.tcp_mem = 65536 131072 262144
net.ipv4.udp_mem = 65536 131072 262144

# Increase the read-buffer space allocatable
net.ipv4.tcp_rmem = 8192 87380 16777216
net.ipv4.udp_rmem_min = 16384
net.core.rmem_default = 131072
net.core.rmem_max = 16777216

# Increase the write-buffer-space allocatable
net.ipv4.tcp_wmem = 8192 65536 16777216
net.ipv4.udp_wmem_min = 16384
net.core.wmem_default = 131072
net.core.wmem_max = 16777216

# Increase number of incoming connections
net.core.somaxconn = 32768

# Increase number of incoming connections backlog
net.core.netdev_max_backlog = 4096
net.core.dev_weight = 64

# Increase the maximum amount of option memory buffers
net.core.optmem_max = 65536

# Increase the maximum number of skb-heads to be cached
#net.core.hot_list_length = 1024

# Increase the tcp-time-wait buckets pool size to prevent simple DOS attacks
net.ipv4.tcp_max_tw_buckets = 1440000
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1

# Limit number of orphans, each orphan can eat up to 16M (max wmem) of unswappable memory
net.ipv4.tcp_max_orphans = 16384
net.ipv4.tcp_orphan_retries = 0

# Increase the maximum memory used to reassemble IP fragments
net.ipv4.ipfrag_high_thresh = 512000
net.ipv4.ipfrag_low_thresh = 446464

# don't cache ssthresh from previous connection
net.ipv4.tcp_no_metrics_save = 1
net.ipv4.tcp_moderate_rcvbuf = 1

# Increase RPC slots
sunrpc.tcp_slot_table_entries = 32
sunrpc.udp_slot_table_entries = 32

# Increase size of RPC datagram queue length
net.unix.max_dgram_qlen = 50

# Don't allow the arp table to become bigger than this
net.ipv4.neigh.default.gc_thresh3 = 2048

# Tell the gc when to become aggressive with arp table cleaning.
# Adjust this based on size of the LAN. 1024 is suitable for most /24 networks
net.ipv4.neigh.default.gc_thresh2 = 1024

# Adjust where the gc will leave arp table alone - set to 32.
net.ipv4.neigh.default.gc_thresh1 = 32

# Adjust to arp table gc to clean-up more often
net.ipv4.neigh.default.gc_interval = 30

# Increase TCP queue length
net.ipv4.neigh.default.proxy_qlen = 96
net.ipv4.neigh.default.unres_qlen = 6

# Enable Explicit Congestion Notification (RFC 3168), disable it if it doesn't work for you
net.ipv4.tcp_ecn = 1
net.ipv4.tcp_ecn = 2
net.ipv4.tcp_reordering = 3

# How many times to retry killing an alive TCP connection
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_retries1 = 3

# This will enusre that immediatly subsequent connections use the new values
net.ipv4.route.flush = 1
net.ipv6.route.flush = 1



###
### Comments/suggestions/additions are welcome!
###

Tuning network and vm settings on CentOS Linux servers with ktune

While poking around the CentOS package repository, I came across the ktune package. Ktune comes with a set of kernel tunables that are useful for network and disk intensive workloads, and provides the ktune service to apply these settings during system startup. Ktune includes settings for TCP/IP buffers, setting the deadline scheduler as the default I/O scheduler, and entries to adjust the swappiness, dirty_ratio and pagecache settings. The full list of tunables can be viewed by paging through the following two configuration files:
$ less /etc/sysctl.ktune
$ less /etc/sysconfig/ktune
To activate the settings, you can enable the ktune service with the chkconfig and service utilities:
$ chkconfig ktune on
$ service ktune start

Saving current sysctl settings:                            [  OK  ]
Applying ktune sysctl settings from /etc/sysctl.ktune:     [  OK  ]
Applying sysctl settings from /etc/sysctl.conf:            [  OK  ]
Applying deadline elevator: sda                            [  OK  ]

This is an awesome package, and I definitely plan to use the network settings on all of my CentOS hosts.

Courtesy : http://prefetch.net/blog/index.php/2009/04/26/performance-tuning-linux-servers-with-ktune/

Monitoring Linux server performance with procallator

I manage a fair number of Linux hosts, and like to keep tabs on how my systems are performing. One way I accomplish this is with procallator, which is a Perl script that collects performance data that can be graphed by orca. The graphs that orca produces are great awesome for trening server performance over time, and can be extremely valuable when debugging performance problems.
To setup procallator to collect performance data, you first need to retrieve the latest orca CVS snapshot from the orcaware snapshots directory (the procallator script is included with the orca snapshot, and the latest version contains a number of fixes). Once orca is downloaded, you will need to extract the tarball and run configure to modify the variables in the header of the procallator script:
$ tar xfj orca-snapshot-r529.tar.bz2
$ cd orca-snapshot-r529
$ ./configure –prefix=/opt/orca-r529 –with-html-dir=/opt/html
After the configure operation completes, you can install the procallator scripts with the Makefile’s install option:
$ make install
This will place the procallator perl script in $PREFIX/bin. To make sure the script starts at system boot, you can copy the $PREFIX/data_gathers/procallator/S99procallator script to /etc/rc3.d (or /etc/init.d depending on how you install your init scripts):
$ cp S99procallator /etc/rc3.d
Once these files are in place, you can start procallator by invoking the init script with the start option:
$ /etc/rc3.d/S99procallator start
This will start the procallator script as a daemon process, and the script will write performance data to the directory defined in the procallator script’s DEST_DIR variable every 5 minutes (this is tunable). The performance files will contain the name proccol-YYYY-MM-DD-INDEX, and one file will be produced each day. To graph the data in the procallator files, you can use orca and the procallator.cfg file that is in the $PREFIX/data_gathers/procallator directory. I placed a sample set of performance graphs on my website, and you can reference the article monitoring LDAP performance article for details on setting up orca to graph data. I digs me some procallator!

Courtesy : http://prefetch.net/blog/index.php/2007/06/15/monitoring-linux-server-performance-with-procallator/

Using LoadAvg for Performance Optimization

Linux and Unixes have excellent metric of system load called “loadavg”. In fact load average is is 3 numbers which correspond to “load average” calculated for one five and 15 minutes. It is computed as exponential moving average so most recent load have more weight in the value than old one.

What does Load Average corresponds to ? At least on Linux it is number of processes which are in “running” state or in “uninterruptable sleep” state which typically corresponds to disk IO. You can also map LoadAvg to VMSTAT output – it is something like moving average of sum of “r” and “b” columns from VMSTAT.

Obviously minimum value for LoadAvg is zero which corresponds to completely idle system, and there is no maximum :)

First thing to understand about LoadAvg it does not really tell you if it is CPU bound load or IO bound load. For example if you have LoadAvg of 10 it may mean there are 10 processes/threads actively consuming CPU or it could be same 10 processes waiting on disk IO and you can see CPU utilization being close to zero.

Second thing is to understand LoadAvg values are relative to your system size. If you have single CPU and 1 disk loadavg of 2 can be considered significant, while if you have 16 CPUs and 2 disks Load of 4 can be light if it is CPU bound – because the system can execute much more CPU bound tasks in parallel or High if it is Disk Bound LoadAvg.

Low Load Average does not mean there are no performance problems, for example if you run single batch job on the server with MySQL, Load Average is likely to be close to 1 even if there are a lot of CPUs and Disks – system may be quite idle and performance still poor because application is not parallel enough. Similar situations can happen if there is a lot of network IO involved or if there are a lot of locks (table/row level locks) or other limiting factors such as innodb_thread_concurrency.

The most interesting question I think is how LoadAvg represent box load in terms of how much load it can handle before it becomes to slow down or being completely unable to handle the load, and it is tricky question. Both for CPUs and for Disk there are two stages request can be. It can be ether currently executing or queued for further execution. The time which is needed to complete request is sum of time it was really executed and the time it was spent in queue. As the system is loaded response time starts to increase mainly because of time requests spend waiting in various queues and waiting on locks, the time of true execution may well remain constant. This is a bit of simplifications as there are number of other effects coming in play but good enough for sake of explanation.

What does it mean from LoadAvg standpoint ? You need to understand where parallel execution continues and where waiting in the queue starts. If you have fully CPU bound workload which is rather parallel (ie many queries will run at once) and you have 4CPUs until your LoadAvg is below 4 you have low time spend waiting for CPUs to be free to do the work. There is some wait but not much. So if you have LoadAvg of 1 and your workload scales linearly with number of connections and CPUs (ie there are no row waits involved) you can assume box can handle up to 3-4 times more load before response time starts to suffer.

If however the LoadAvg is 4 already it may take rather insignificant increase to take it up to 8 and you will see some delays due to queuing. If there are 4CPUs (Cores) and loadavg is 16 for CPU bound workload it often means requests should take 4 times more to complete than they would on idle box due to waiting in the queue.

Same true for pure Disk IO bound workload with small difference of disk not being replaceable (if you’re waiting on one drive you can’t use another drive instead), and the fact disks can optimize multiple outstanding requests a bit better compared to requests coming one after another.

For mixed workload, which is what we usually see in practice you have to do some assumptions guesses or further analyzes if you want good estimates. Ie you may want to check mpstat, vmstat and iostat to see where load comes from. But the general rule remains the same – until you’re able to explore parallel abilities of the box it will perform well as soon as you need to do a lot of queuing performance starts to suffer.

Let us clarify last point – how much more load the box can handle before it overloads, loadavg skyrockets and it becomes as good as down. First for many applications request inflow is not constant – ie web site gets poor response time and users do not spend so much time on it any more so load drops. This is however temporary relive only as there are stubborn users which would not go away even with slow responding site until their browsers timeout, which is as good as site is down. There are too many variables to come with exact numbers but generally as soon as you have long queuing started it may take just 10-20% extra load to overload system, so it is better to keep loadavg low – below number of CPUs and/or disks you have.

I must note – LoadAvg is not perfect tool for the task. It is just almost always available unlike other metrics. It is best to have profiling information so you can see as response time for your requests starts to grow. As soon as it becomes to grow with no good reason I would start to worry whatever LoadAvg shows.

Courtesy : http://www.mysqlperformanceblog.com/2006/12/04/using-loadavg-for-performance-optimization/

Basic I/O Monitoring on Linux

This is my fourth week at Pythian and in Canada and I’m starting to get back to my normal life cycle — my personal things are getting sorted and my working environment is set. Here at Pythian I’m in a team of four people together with Christo, Joe, and Virgil. (I should write another post about beginning at Pythian — will do one day.)
Yesterday, I asked Christo to show me how he monitors I/O on Linux. I needed to collect statistics on a large Oracle table on a production box, and wanted to keep an eye on the impact. So we grabbed Joe as well and sat all three around my PC. While we were discussing, Paul was around and showed some interest in the topic we discussed — otherwise, why would we all three be involved?. Anyway, Dave and Paul thought that this would be a nice case for a blog post. So here we are…
Indeed, while the technique we discuss here is basic, it gives a good overview and is very easy to use. So let get focused… We will use iostat utility. In case you need you know where to find more about it — right, man pages.
So we will use the following form of the command:

iostat -x [-d] <interval>

• -x option displays extended statistics. You definitely want it.
• -d is optional. It removes CPU utilization to avoid cluttering the output. If you leave it out, you will get the following couple lines in addition:

  avg-cpu:  %user   %nice    %sys %iowait   %idle    6.79    0.00    3.79   16.97   72.46

• is the number of seconds iostat waits between each report. Without a specified interval, iostat displays statistics since the system was up then exits, which is not useful in our case. Specifying the number of seconds causes iostat to print periodic reports where IO statistics are averaged for the time period since previous report. I.e., specifying 5 makes iostat dump 5 seconds of average IO characteristics, every 5 seconds until it’s stopped.

If you have many devices and you want to watch for only some of them, you can also specify device names on command line:

iostat -x -d sda 5

Now let’s get to the most interesting part — what those cryptic extended statistics are. (For readability, I formatted the report above so that the last two lines are in fact a continuation of the first two.)

Device:    rrqm/s wrqm/s   r/s   w/s  rsec/s  wsec/s    rkB/s sda          0.00  12.57 10.18  9.78  134.13  178.84    67.07   wkB/s avgrq-sz  avgqu-sz   await  svctm  %util 89.42    15.68      0.28   14.16   8.88  17.72

• r/s and w/s— respectively, the number of read and write requests issued by processes to the OS for a device.
• rsec/s and wsec/s — sectors read/written (each sector 512 bytes).
• rkB/s and wkB/s — kilobytes read/written.
• avgrq-sz — average sectors per request (for both reads and writes). Do the math — (rsec + wsec) / (r + w) = (134.13+178.84)/(10.18+9.78)=15.6798597
  If you want it in kilobytes, divide by 2.
  If you want it separate for reads and writes — do you own math using rkB/s and wkB/s.
• avgqu-sz — average queue length for this device.
• await — average response time (ms) of IO requests to a device. The name is a bit confusing as this is the total response time including wait time in the requests queue (let call it qutim), and service time that device was working servicing the requests (see next column — svctim). So the formula is await = qutim + svctim.
• svctim — average time (ms) a device was servicing requests. This is a component of total response time of IO requests.
• %util — this is a pretty confusing value. The man page defines it as, Percentage of CPU time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100%. A bit difficult to digest. Perhaps it’s better to think of it as percentage of time the device was servicing requests as opposed to being idle. To understand it better here is the formula: utilization = ( (read requests + write requests) * service time in ms / 1000 ms ) * 100%
  or
  %util = ( r + w ) * svctim /10 = ( 10.18 + 9.78 ) * 8.88 = 17.72448

Traditionally, it’s common to assume that the closer to 100% utilization a device is, the more saturated it is. This might be true when the system device corresponds to a single physical disk. However, with devices representing a LUN of a modern storage box, the story might be completely different.
Rather than looking at device utilization, there is another way to estimate how loaded a device is. Look at the non-existent column I mentioned above — qutim — the average time a request is spending in the queue. If it’s insignificant, compare it to svctim — the IO device is not saturated. When it becomes comparable to svctim and goes above it, then requests are queued longer and a major part of response time is actually time spent waiting in the queue.
The figure in the await column should be as close to that in the svctim column as possible. If await goes much above svctim, watch out! The IO device is probably overloaded.
There is much to say about IO monitoring and interpreting results. Perhaps this is only the first of a series of posts about IO statistics. At Pythian we often come across different environments with specific characteristics and various requirements that our clients have. So stay tune — more to come.

Courtesy : http://www.pythian.com/news/247/basic-io-monitoring-on-linux/

Measuring & Optimizing I/O Performance

Measuring and optimizing IO performance is somewhat of a black art: the tools are there, the resources and discussions are plenty, but it is also incredibly easy to get lost in the forest. I speak from recent experience. Having gone down multiple false starts with filesystem optimization, RAID tweaking, and even app-level changes it really helped to finally step back and revisit the basics. Many man pages and discussion threads later, a few useful realizations emerged: iostat is your best friend, but it can also be incredibly deceiving; refreshing your memory of disk latencies will go a long way; disks and filesystems are fast, but not that fast.

Monitoring IO Performance with iostat

If IO performance is suspect, iostat is your best friend. Having said that, the man pages are cryptic so don't be surprised if you find yourself reading the source. To get started, identify the device in question and start a monitoring process:

# -k output rates in kB
# -x output extended stats
# -d monitoring single device
# sample stats every 5 seconds for device /dev/sdh
[ilya@igvita]> iostat -dxk /dev/sdi 5

Next, allocate yourself a couple of hours to understand the output or expect to find yourself down a wrong path in no time flat (been there, done that). iostat is a popular tool amongst the database crowd, so not surprisingly you'll find a lot of great discussions documenting the use. Depending on your application you will need to focus on different metrics, but as a gentle introduction let's take a look at await, svctime and avgque:

await - The average time (in milliseconds) for I/O requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.
svctime - The average service time (in milliseconds) for I/O requests that were issued to the device.
avgqu-sz - The average queue length of the requests that were issued to the device.

First off, await is a deceiving metric! Even though it claims to measure average time, it is better understood as an aggregate function, so don't be mislead by it: avgqu-sz * svctm / (%util/100). Ideally, await should be roughly equal to your svctime, which leads us to a corollary: your average queue size is ideally hovering around single digits. Understanding these variables alone can tell you volumes about the application generating the load.

Disk Latencies Refresher & EBS Performance

Disk access time is determined via the sum of several variables: spin-up, seek, rotational delay, and transfer time. Assuming your disk is not is not sleeping we can discount the spin-up time, which leaves us with seek (time for the disk arm to find the track: ~10ms), rotational delay (time to get the right sector under the head: depends on RPM), and the actual transfer time. Hence, in the worst case we will take ~10ms to seek, 60s/7200RPM ~= 8ms in rotational delay, plus the read time. On average, for a 7.2k RPM disk this translates into roughly ~5ms access time (~20ms in worst case) to read the first byte!
Armed with this knowledge we can now put Amazon's EBS performance in context: on average our EBS mounts show 10~30ms svctime, which all things considered is not outrageous for a SAN. This number also dips into low single digits at nights and on weekends, which points to the fact that as with any shared resource, the performance of EBS degrades during the day. Having said that, a 6x performance difference based on time of day is definitely not anything to sneeze at, so let's hope Amazon is on top of this!
Average queue size (avgqu-sz) is a popular metric in the DBA circles, but do be careful with it when running on a SAN or any multi-spindle device. Ideally, your queue size (avgqu-sz) for a single disk should be in single digits, which means that the underlying device is well matched to the IO load generated by the application. Conversely, if the queue size is artificially low, chances are your application code can benefit from some tuning: do less disk flushing, think about caching or buffering, or in other words, double check the assumption that IO is the bottleneck!

Disks, Filesystems and Facebook Case Study: Haystack

Average access time on our disks places some hard limits on the number of IOPs - at 5ms average, we get a very optimistic 200 req/s with no read time. Hence, if you're trying to store several hundred files a second, you might want to revisit the architecture or seriously think about switching to SSD's! Databases such as MySQL work around this constraint by minimizing the number of file handles, caching data, and using aggressive buffering techniques. Willing to potentially loose a little bit of data with InnoDB? Set flush_log_at_trx_commit to 2 to avoid flushing on every transaction in favor of a periodic one second flush. In similar fashion, you can tweak your MyISAM key buffers, or even place your index and data files on different drives.
Facebook team recently released the details of their Haystack photo storage system which serves as a great case study of working around the IO bottlenecks: over 15PB of photo storage, and ~360 new photos being uploaded every second as of April '09. To meet the requirements, they dropped the POSIX filesystem semantics and went for an append only structure with a separate in-memory index which stores the direct inode offsets for each photo. As a result, each photo access is translated into a single IO request - a huge win. Read through it, fascinating stuff and an illustrative example of optimizing for IO.

Courtesy : http://www.igvita.com/2009/06/23/measuring-optimizing-io-performance/

Redirect Users To A Fake Speed Test Site

For some ISPs, speed test sites can be their best friend. It shows a user that he is reaching his potential speed. For others it’s a nightmare as it gives ammunition for users. While at MUM this year my lunch table was having a friendly discussion about just this. Some suggestions were offered on various things you can do to speed test sites. One person explained how a WISP was blocking every speed test site they could think of, just to keep users from accessing any solid material to complain with. I then suggested that I could make a fake speed test site and redirect all users to that. Everyone had a good laugh, but…I though it would be fun to do it for real, so here it is!
In a nutshell what happens is this:

The user tries to browse to a webpage that contains the word speedtest or bandwidth test in the url

The server IP gets added to an address list of known speed test servers.

Subsiquent access to these addresses hit a destination nat rule that will shoot them over to our webserver, which happens to be hosting our custom webpage.

Layer 7 match for an URL that contains speedtest or bandwidthtest.

1 2 3

/ip firewall layer7-protocol add comment="" name=speedtest-url regexp=\ "^.*(get|GET).+/(speed|bandwidth).*test.*\$"

Here’s the mangle rule to mark speed test traffic.

1 2 3 4 5

/ip firewall mangle add action=add-dst-to-address-list address-list=speedtest-al \ address-list-timeout=0s chain=prerouting comment=\ "speedtest mark connection" disabled=no dst-port=80 \ layer7-protocol=speedtest-url protocol=tcp

Here’s the NAT statement to grab the traffic and send it to our webserver.
Our webserver is 1.1.1.1 in this example.

1 2 3

/ip firewall nat add action=dst-nat chain=dstnat comment="" disabled=no dst-address-list=\ speedtest-al to-addresses=1.1.1.1

Courtesy : http://mikrotikuniversity.com/index.php/blog/