It is perfectly possible to configure a linux server (or workstation if you wish) to talk IPsec. The Linux Advanced Routing & Traffic Control site has a page describing it. Since IPsec is a standard protocol, I wanted to get a tunnel up and running between a linux host and a Cisco router: with success! Here are the config files that I used in this test:

racoon.conf:

path pre_shared_key "/etc/racoon/psk.txt";
path certificate "/etc/racoon/certs";

remote anonymous {
	exchange_mode main;
	my_identifier address 172.31.8.102;
	verify_identifier on;
	proposal_check strict;
	proposal {
		encryption_algorithm aes;
		hash_algorithm sha1;
		authentication_method pre_shared_key;
		dh_group modp1536;
		lifetime time 24 hours;
	}
}
remote 172.31.8.10 inherit anonymous {
	peers_identifier address 172.31.8.10;
}
sainfo anonymous {
	pfs_group modp1536;
	lifetime time 1 hour;
	encryption_algorithm aes 128;
	authentication_algorithm hmac_sha1;
	compression_algorithm deflate;
}

psk.txt:

172.31.8.10   presharedkey

setkey commands:

sdpflush;
flush;

spdadd 172.31.8.102 172.31.8.10 -P out ipsec esp/transport//require;
spdadd 172.31.8.10 172.31.8.102 -P in ipsec esp/transport//require;

Cisco config (relevant parts):

crypto isakmp policy 1
 encr aes
 authentication pre-share
 group 5
crypto isakmp key presharedkey address 172.31.8.102
!
crypto ipsec transform-set TS_ESP_AES_SHA esp-aes esp-sha-hmac
 mode transport
!
crypto map CM_TEST 1 ipsec-isakmp
 set peer 172.31.8.102
 set transform-set TS_ESP_AES_SHA
 set pfs group5
 match address ACL_IPSEC
!
interface FastEthernet0/0
 ip address 172.31.8.10 255.255.255.0
 duplex auto
 speed auto
 crypto map CM_TEST
!
interface FastEthernet0/1
 no ip address
 shutdown
 duplex auto
 speed auto
!
ip access-list extended ACL_IPSEC
 permit ip host 172.31.8.10 host 172.31.8.102